package main.pmc;

import cn.hutool.core.util.ZipUtil;
import cn.hutool.db.Db;
import cn.hutool.db.Entity;
import com.alibaba.fastjson.JSONObject;
import lombok.SneakyThrows;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import utils.SmallTool;
import utils.StringUtil;

import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static main.pmc.LocalPmcFileCheck.isCheckDownType;

/**
 * @Author: gcr
 * @Date: 2022/3/12 13:20
 */
public class DownPmcidHtmlParseTest {
    // 用于下载pmcid的页面保存


    public static String dealUrl(String url) {
        if (!url.startsWith("http")) {
            return ("https://pmc.ncbi.nlm.nih.gov/" + url);
        } else {
            return url;
        }
    }

    @SneakyThrows
    public static void main(String[] args) {
//        DSFactory.setCurrentDSFactory(new DruidDSFactory());
//        DataSource ds = DSFactory.get("taiyiserver");
        Db useTaiyi = Db.use();
        String saveTableFrom = "pmcdownziphtml";
        String saveTableDwon = "pmcdown_new2";
        String allDate = "select max(pmid) as pmid  from " + saveTableFrom;
        Entity query = useTaiyi.query(allDate).get(0);
        SmallTool.printMess("最大的id 是" + query.getStr("pmid"));
        Integer count = query.getInt("pmid");
        int breakSize = 100000;
        int pageSize = count / breakSize + 1;
        SmallTool.printMess("全部页码+++++++++" + pageSize);
        SmallTool.printMess("任务开始+++++++++++++++++++++++++++++++++++++++++++++++++++++");
        for (int i = 1; i <= pageSize; i++) {
            // 这里写错了 需要只需要入库特定的数据 其他不要
            int start = breakSize * (i - 1) + 1;
            int end = i * breakSize;
            SmallTool.printMess("当前页码+++++" + i);
            SmallTool.printMess("开始id++" + start + "+++结束id++++" + end);
            String paramSql = "select pmid,pmcid,html from " + saveTableFrom + " where  pmid between  " + start + " and " + end;
//            String paramSql = "select pmid,pmcid,html,supp,down from " + saveTableFrom + " where  pmid  = 39901448 ";
            List<Entity> querys = useTaiyi.query(paramSql);
            querys.parallelStream().forEach(entity -> {
                String pmcid = entity.getStr("pmcid");
                String pmid = entity.getStr("pmid");
                byte[] htmls = entity.getBytes("html");
//                    String html = ZipUtil.unGzip(entity.getBytes("html"), "utf-8");
                int ssta = 0;
                String q = "select pmid,down from " + saveTableDwon + " where  pmid =  " + pmid;
                try {
                    Entity entity1 = useTaiyi.queryOne(q);
                    if (entity1 != null) {
                        ssta = entity1.getInt("down");
                    }
                } catch (SQLException e) {
                    throw new RuntimeException(e);
                }

                Map<String, SaveEntity> completeMap = new HashMap<>();


//                Set<String> urlSet = new HashSet<>();
                if (StringUtil.isNotNull(htmls) && (ssta == 0)) {
                    String html = ZipUtil.unGzip(htmls, "UTF-8");
                    Document parse = Jsoup.parse(html);
                    Element element1 = parse.selectFirst("section.associated-data");
                    if (element1 != null) {
                        Elements a = element1.select("a");
                        for (Element element : a) {
                            SaveEntity saveEntity = new SaveEntity();
                            String href = dealUrl(element.attr("href"));
                            boolean downUrl = isCheckDownType(href);
                            if (downUrl) {
                                Element element22 = element.nextElementSibling();
                                if (element22 != null) {
                                    String text = element22.text();
                                    saveEntity.setSize(text);
                                    saveEntity.setUrl(href);
                                    completeMap.put(href, saveEntity);
                                }
                            }
                        }
                    }
                    Elements element2 = parse.select("h2:contains(Supplementary Material)");
                    if (element2 != null) {
                        for (Element elementParam : element2) {
                            Element parent = elementParam.parent();
                            Elements a = parent.select("a");
                            for (Element element : a) {
                                SaveEntity saveEntity = new SaveEntity();
                                String href = dealUrl(element.attr("href"));
                                boolean downUrl = isCheckDownType(href);
                                if (downUrl) {
                                    Element element22 = element.nextElementSibling();
                                    if (element22 != null) {
                                        String text = element22.text();
                                        saveEntity.setSize(text);
                                        saveEntity.setUrl(href);
                                        completeMap.put(href, saveEntity);
                                    }
                                }
                            }
                        }

                    }
                    // Appendix. Supplementary materials
                    Element element3 = parse.selectFirst("h2:contains(Appendix. Supplementary materials)");
                    if (element3 != null) {
                        Element parent = element3.parent();
                        Elements a = parent.select("a");
                        for (Element element : a) {
                            SaveEntity saveEntity = new SaveEntity();
                            String href = dealUrl(element.attr("href"));
                            boolean downUrl = isCheckDownType(href);
                            if (downUrl) {
                                Element element22 = element.nextElementSibling();
                                if (element22 != null) {
                                    String text = element22.text();
                                    saveEntity.setSize(text);
                                    saveEntity.setUrl(href);
                                    completeMap.put(href, saveEntity);
                                }
                            }
                        }
                    }
                    // Abbreviations
                    Element element4 = parse.selectFirst("div:contains(Multimedia Appendix)");
                    if (element4 != null) {
                        Element parent = element4.parent();
                        Elements a = parent.select("a");
                        for (Element element : a) {
                            SaveEntity saveEntity = new SaveEntity();
                            String href = dealUrl(element.attr("href"));
                            boolean downUrl = isCheckDownType(href);
                            if (downUrl) {
                                Element element22 = element.nextElementSibling();
                                if (element22 != null) {
                                    String text = element22.text();
                                    saveEntity.setSize(text);
                                    saveEntity.setUrl(href);
                                    completeMap.put(href, saveEntity);
                                }
                            }
                        }
                    }


                    int size = completeMap.size();

                    if (size > 0) {
                        Entity ins = new Entity();
                        ins.setTableName(saveTableDwon);
                        ins.set("pmid", pmid);
                        List<SaveEntity> ls = new ArrayList<>();
                        completeMap.forEach((k, v) -> {
                            ls.add(v);
                        });
                        ins.set("supp", JSONObject.toJSONString(ls));
                        ins.set("down", 1);
                        try {
                            useTaiyi.insert(ins);
                        } catch (SQLException e) {
                            throw new RuntimeException(e);
                        }
                    } else {
                        Entity ins = new Entity();
                        ins.setTableName(saveTableDwon);
                        ins.set("pmid", pmid);
//                        ins.set("supp", JSONObject.toJSONString(ls));
                        ins.set("down", 3);
                        try {
                            useTaiyi.insert(ins);
                        } catch (SQLException e) {
                            throw new RuntimeException(e);
                        }
                    }

                }
            });

        }
    }
}
